import requests
from bs4 import BeautifulSoup
from tqdm import tqdm
import time
import pandas as pd


from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all" 

#https://www.coincola.com/ad-detail/17001-19999
trade_urls = ['https://www.coincola.com/ad-detail/%d' % (17000+i) for i in range(20000)]

#由于网站禁止爬虫，可以在请求加上头信息，伪装成浏览器访问.
headers = {
    'User-Agent': 'Mozilla/5.0'
}


def get_product_info(url):
    
    content = ''
    title = ''
    try:
         r = requests.get(url, headers=headers)
         r.raise_for_status() #如果响应状态码不是 200，就主动抛出异常
    except requests.RequestException as e:
         pass
    else:
        soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器
    
        try:
            content = soup.find('p', class_="p mb-20")
            title = content.get_text()
        except (AttributeError,TypeError):
            pass
     
    return title

def is_number(s):
    try:
        int(s)
        return True
    except ValueError:
        pass
 
    try:
        import unicodedata
        unicodedata.numeric(s)
        return True
    except (TypeError, ValueError):
        pass
 
    return False

def get_phone(strphone):
    #get phone number from ads
    adslen = len(strphone)
 
    getphone = '' 
    for i in (range(adslen)):
   
        ch = strphone[i:i+1]
        if  (ch == '1' and adslen-i>10):
			
            strRightString = strphone[i:11+i]
            if (is_number(strRightString)):
				#IS PHONE
                getphone = strRightString
                break
            else:
                continue
        
    return getphone
    
ads_data = []
phone_data = []

for detail_url in tqdm(trade_urls):
    detail_ads = get_product_info(detail_url)
    ads_data.append(detail_ads)
    
    myphone = get_phone(detail_ads)
    phone_data.append(myphone)
		

table=pd.DataFrame({'ads':ads_data,'phone':phone_data})

date = time.strftime('%Y%m%d',time.localtime(time.time()))

table.to_csv(r'data/coincola%s.csv' % (date),encoding = "utf_8_sig")
